#获取网页
import requests
import json
from sqlalchemy_helper import *


def get_page(page, action):
    # url = 'https://list.mogu.com/search?callback=jQuery21105713808137705516_1581471939424&_version=8193&ratio=3%3A4&cKey=15&page='+str(page)+'&sort=pop&ad=0&fcid=50240&action=' + action + '&acm=3.mce.1_10_1ko4s.132244.0.zudQarQfjfLsQ.pos_1-m_482170-sd_119&ptp=31.v5mL0b._head.0.ibjb3sk8&_=1581471939426'
    # https://list.mogu.com/search?callback=jQuery21109400356421341389_1581488095758&_version=8193&ratio=3%3A4&cKey=15&page=1&sort=pop&ad=0&fcid=51716&action=boyfriend&acm=3.mce.1_10_1ko5c.132244.0.heoPzrQfl6uHq.pos_7-m_482180-sd_119&ptp=31.Onv5v._head.34.EmfnrZ5B&_=1581488095759
    url ='https://list.mogu.com/search?&cKey=15&page='+str(page)+'&action='+action+''

    #伪造请求头
    headers = {
        "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.130 Safari/537.36"
    }
    respose = requests.get(url, headers=headers)
    if respose.status_code == 200:
        # respose.conent返回的是字节流，用decode转换成字符串
        return respose.content.decode('utf-8')
    return None

# 解析网页
def parse_page(html, action):
    # start_index = html.index('(')
    # html = html[start_index + 1:-2]
    #print(html)
    json_data = json.loads(html)
    #获取是否结束的标记
    is_end = json_data['result']['wall']['isEnd']
    result = json_data['result']['wall']['docs']
    for item in result:
        # 放类别进去
        item['category'] = action
        print(item['title'])
        # 用sqlalchemy保存打数据库中
        save_goods(item)
    return is_end
#获取所有的数据
def get_all_page():

    #actions = ['clothing', 'skirt', 'trousers', 'shoes', 'boyfriend', 'neiyi', 'bags', 'baby', 'home']
    actions = ['clothing', 'neiyi']

    for action in actions:
        page = 1
        print(action)
        print( '*' * 20)
        while True:
            print(page)
            html = get_page(page, action)
            is_end = parse_page(html, action)
            if is_end:
                break
            page+=1


def main():
    get_all_page()


if __name__ == '__main__':
    main()

